//g++ mongohlp.cc -lboost_program_options
#include <iostream>
#include <fstream>
#include <algorithm>
#include <vector>
#include <boost/functional/hash.hpp>
#include <google/sparse_hash_map>
#include <boost/program_options.hpp>
#include <re2/re2.h>
#include <stdio.h>
#include <string>
#include <stdlib.h>
#include "client/dbclient.h"
using namespace std;
using namespace mongo;

typedef google::sparse_hash_map<string, int, boost::hash<string> > Dict;
typedef Dict::value_type Item;


using namespace re2;
class word_filter{
     private:
          vector<string> m_filter_words;
          string  m_filter_pattern;
     public:
          char* trim( char* str, size_t size ){
               if( str[size-2] == '\r' ){
                    str[size-2] = 0;
               } else if( str[size-1] == '\n'){
                    str[size-1] = 0;
               }
               return str;
          };
          void join(vector<string>& filter_words,string& word_string  )
          {
               size_t i = 0;
               for( ; i< filter_words.size()-1; i++ ){
                    word_string += filter_words[i]+"|";
               }
               word_string+=filter_words.back();
          }
          void load_filter(const char* filter_file){
               FILE *fp;
               char *line = NULL;
               size_t len = 0;
               ssize_t read;

               fp = fopen(filter_file, "r");
               if (fp){
                    while ((read = getline(&line, &len, fp)) != -1) {
                         m_filter_words.push_back(trim(line,read));
                    }
                    free(line);
               }
          }
          word_filter( const char* filter_file ){
               load_filter( filter_file );
               join(m_filter_words,m_filter_pattern);
          }
          string& get_pattern(){ return  m_filter_pattern;}
          vector<string>& get_filter_words(){
               return m_filter_words;
          }
}; //word_filter
word_filter filter("filters.txt");
namespace po = boost::program_options;
struct cmd_line { 
     cmd_line() : port(DefaultDBPort),limit(0) { } 

     enum { 
          DefaultDBPort = 27017,
     };

     int port;
     int limit;

     string collection;
     string host;
     string field;
};
int parse_cmdline( cmd_line &cmd_line,int argc,char*argv[])
{
     bool error=false;
     boost::program_options::options_description options("command line options");
     boost::program_options::variables_map option_map;
     options.add_options() 
          ("help,h", "show this usage information")
          ("version", "show version information")
          ("host,h", po::value<string>(), "source server")
          ("port,p", po::value<int>(&cmd_line.port), "mongo port (default 27017)")
          ("limit,n", po::value<int>(&cmd_line.limit), "count limit to process")
          ("field,f", po::value<string>(), "source server")
          ("coll,c", po::value<string>(), "collection name(for example,\"zbot_development.download\")");
     try {
          boost::program_options::store( boost::program_options::parse_command_line(argc, argv, options), option_map);
          boost::program_options::notify(option_map);

          if (option_map.count("help")) {
               cout << options << endl;
          }
          const char *required[]={"host","field","coll"};
          const char *required_desc[]={"mongodb host","field of collection","collection name"};
          string* params[]={&cmd_line.host,&cmd_line.field,&cmd_line.collection};
          for( size_t i=0 ; i< sizeof(required)/sizeof(char*); i++ ){
               if( !option_map.count(required[i]) ) {
                    error= true;
                    cout << required_desc[i] << " required" << endl;
               } else {
                    params[i]->append(option_map[required[i]].as<string>());
               }
          }
          if( error ){
               cout << options << endl;
          }
          if( option_map.count("port") ) {
               cout << cmd_line.port << endl;
               if(!( cmd_line.port > 1024 && cmd_line.port < 65536) ){
                    cout << options << endl;
                    error = true;
               }
          }
     } catch(po::error &e){
          cout << "ERROR: " << e.what() << endl << endl;
          cout << options << endl;
          return false;
     }
     return !error;

}
bool filter_word( const char* word )
{
     if(RE2::PartialMatch(word, filter.get_pattern())) {
          return true;
     }
     return false;
}
bool compareWordCount(const Item *a, const Item *b)
{
     return a->second > b->second;
}
void print_result(  Dict& words ) 
{
     ofstream out;
     out.open("count_word.txt");
     vector<Item*> vec;
     vec.reserve(words.size());
     for(Dict::iterator i = words.begin(); i != words.end(); i++)
     {
          vec.push_back(&(*i));
     }
     sort(vec.begin(), vec.end(), compareWordCount);
     for(vector<Item*>::iterator i = vec.begin(); i != vec.end(); i++)
     {
          cout << (*i)->first << " : " << (*i)->second << endl;
          if (out.is_open())
               out << (*i)->first << " : " << (*i)->second << endl;
     }
}
void count_word( cmd_line &cmd_line )
{
     int count = 0;
     BSONObjBuilder builder;
     builder.append("page_type",2);
     BSONObj query = builder.obj();
     DBClientConnection conn;
     char conn_str[128];
     Dict words;

     sprintf(conn_str,"%s:%d",cmd_line.host.c_str(),cmd_line.port);
     conn.connect(conn_str); 

     BSONObj doc;
     auto_ptr<DBClientCursor> cursor = conn.query(cmd_line.collection, query,cmd_line.limit);

     cout << "counting \t" << cmd_line.collection << "." << cmd_line.field << "@" << conn_str << endl ;
     while( cursor->more() )
     {
          const char *atext = (cursor->next()).getStringField(cmd_line.field.c_str());
          if( strlen(atext) &&  filter_word(atext) or ){
               continue;
          }
          words[string(atext)]++;
          if (++count % 100000 == 0)
          {
               cout << count << " words:" << words.size() << endl;
          }
     }
     cout << "count over:" << words.size() << " words" << endl;
     print_result(words);
}
int main (int argc, char* argv[])
{

     cmd_line cmd_line;
     if( parse_cmdline(cmd_line,argc,argv) ){
          count_word(cmd_line);
     }
     return 0;
}

